package com.chance.cc.crawler.development.bootstrap.xiaohongshu;

import com.alibaba.fastjson.JSON;
import com.chance.cc.crawler.core.downloader.HttpConfig;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.downloader.HttpRequest;
import com.chance.cc.crawler.core.downloader.HttpRequestBody;
import com.chance.cc.crawler.core.downloader.http.HttpClientDownloader;
import com.chance.cc.crawler.core.downloader.http.HttpClientGenerator;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.downloader.proxy.SiteConfigProxyProvider;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.selector.Json;
import com.chance.cc.crawler.development.bootstrap.xcar.XCarHttpTest;
import com.chance.cc.crawler.development.utils.RedisReader;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.NameValuePair;
import org.apache.http.client.utils.URLEncodedUtils;
import org.apache.http.message.BasicNameValuePair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.*;

import static org.codehaus.groovy.runtime.EncodingGroovyMethods.md5;

/**
 * @author bx
 * @date 2021/2/18 0018 15:36
 */
public class XHSTest {
    public static Logger logger = LoggerFactory.getLogger(XCarHttpTest.class);

    private static final char[] f52734c = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'};

    private static Proxy proxy = new Proxy();

    static {
        //代理配置
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HL89Q19E86E2987D");
        proxy.setPassword("71F33D94CE5F7BF2");
    }

    static Map<String,String> headers = new HashMap<>();
    static {
        headers.put("charset", "utf-8");
        headers.put("Accept-Encoding", "gzip");
        headers.put("referer", "https://servicewechat.com/wxb296433268a1c654/34/page-frame.html");
//        headers.put("authorization", "wxmp.d6cee3a9-8725-4f32-986e-082355b58d07");
        headers.put("device-fingerprint","WHJMrwNw1k/H00VmAINdX86nnNIFhdYqjvPyLNDxvBdt/Cx4E7aSgzBGsp19lJLLnx7AUAKpfPgQjv5/oh5p5bfQ/QDfAQ8pTdCW1tldyDzmauSxIJm5Txg==1487582755342");
        headers.put("authorization", "wxmp.9be0cebf-b62d-4c87-b17c-bc35b3799014");
        headers.put("content-type", "application/json");
//        headers.put("User-Agent", "Mozilla/5.0 (Linux; Android 6.0; DIG-AL00 Build/HUAWEIDIG-AL00; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/55.0.2883.91 Mobile Safari/537.36 MicroMessenger/7.0.3.1400(0x27000334) Process/appbrand2 NetType/WIFI Language/zh_CN");
//        headers.put("User-Agent", "Mozilla/5.0 (Linux; U; Android 2.2.1; en-us; Nexus One Build/FRG83) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1  MicroMessenger/7.0.3.1400(0x27000334) Process/appbrand2 NetType/WIFI Language/zh_CN");
//        headers.put("User-Agent", "Mozilla/5.0 (Linux; Android 7.1.1; OPPO R9sk) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.111 Mobile Safari/537.36  MicroMessenger/7.0.3.1400(0x27000334) Process/appbrand2 NetType/WIFI Language/zh_CN");
//        headers.put("User-Agent", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36 MicroMessenger/7.0.3.1400(0x27000334) Process/appbrand2 NetType/WIFI Language/zh_CN");
        headers.put("Host", "www.xiaohongshu.com");
        headers.put("Connection", "Keep-Alive");
    }

    static List<String> imageList = new ArrayList<>();
    static {
        imageList.add("https://img.xiaohongshu.com/avatar/6005c142000000000100915c.jpg@80w_80h_90q_1e_1c_1x.jpg");
        imageList.add("https://img.xiaohongshu.com/avatar/5feb7a3e3f9464ee1376bd3f.jpg@80w_80h_90q_1e_1c_1x.jpg");
        imageList.add("https://img.xiaohongshu.com/avatar/6009198ce3c5e954bad9c7c0.jpg@80w_80h_90q_1e_1c_1x.jpg");
        imageList.add("https://img.xiaohongshu.com/avatar/5e421d27000000000100427c.jpg@80w_80h_90q_1e_1c_1x.jpg");
        imageList.add("https://img.xiaohongshu.com/avatar/5ee1d335000000000101de75.jpg@80w_80h_90q_1e_1c_1x.jpg");
        imageList.add("https://img.xiaohongshu.com/avatar/6005c142000000000100915c.jpg@80w_80h_90q_1e_1c_1x.jpg");
        imageList.add("https://img.xiaohongshu.com/avatar/5cc9bba6cbe1220001109594.jpg@80w_80h_90q_1e_1c_1x.jpg");
        imageList.add("https://img.xiaohongshu.com/avatar/5fa2bb62b3ae1200013bd912.jpg@80w_80h_90q_1e_1c_1x.jpg");
        imageList.add("https://img.xiaohongshu.com/avatar/6037b9f2816ab524fa544dc5.jpg@80w_80h_90q_1e_1c_1x.jpg");
        imageList.add("https://img.xiaohongshu.com/avatar/6001b58129f5ee175e4e1c39.jpg@80w_80h_90q_1e_1c_1x.jpg");
        imageList.add("https://img.xiaohongshu.com/avatar/5f7dc2df1573e20001239f70.jpg@80w_80h_90q_1e_1c_1x.jpg");
        imageList.add("https://img.xiaohongshu.com/avatar/5dd0aed5f31df200011fc826.jpg@80w_80h_90q_1e_1c_1x.jpg");
        imageList.add("https://img.xiaohongshu.com/avatar/5ecf5a8c000000000101e7a4.jpg@80w_80h_90q_1e_1c_1x.jpg");

    }

    private static int index = 0;
    static List<String> useragents = new ArrayList<>();
//    static {
//        HttpConfig httpConfig = HttpConfig.me("test");
//        HttpClientDownloader downloader = new HttpClientDownloader();
//        downloader.setClientGenerator(new HttpClientGenerator());
//        downloader.setProxyProvider(new SiteConfigProxyProvider());
//        HttpRequest httpRequest = new HttpRequest();
//        httpRequest.setUrl("https://fake-useragent.herokuapp.com/browsers/0.1.11");
//        HttpPage download = downloader.download(httpRequest, httpConfig);
//        String rawText = download.getRawText();
//        Json json = new Json(rawText);
//        useragents.addAll(json.jsonPath("$.browsers.chrome").all());
//        useragents.addAll(json.jsonPath("$.browsers.opera").all());
//        useragents.addAll(json.jsonPath("$.browsers.firefox").all());
//        useragents.addAll(json.jsonPath("$.browsers.internetexplorer").all());
//        useragents.addAll(json.jsonPath("$.browsers.safari").all());
//    }

    private static int count = 0;
    public static void main(String[] args) throws IOException {

        HttpConfig httpConfig = HttpConfig.me("test");
        httpConfig.setProxy(proxy);
        httpConfig.setUseProxy(true);

        HttpClientDownloader downloader = new HttpClientDownloader();
        downloader.setClientGenerator(new HttpClientGenerator());
        downloader.setProxyProvider(new SiteConfigProxyProvider());

        for (int i = 1; i <= 1; i++) {
            new Thread(new TestSearchRun(downloader, httpConfig), "thread_" + i).start();
        }
//        statics();
//        washKw();
    }

    private static void washKw() throws IOException {
        BufferedReader bufferedReader = new BufferedReader(new FileReader("D:\\chance\\data\\xhs\\xhs_14.log"));
        String line;
        Set<String> kws = new HashSet<>();
        while ((line = bufferedReader.readLine()) != null){
            CrawlerData crawlerData = JSON.parseObject(line, CrawlerData.class);
            String keyword = crawlerData.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("keyword");
            kws.add(keyword);
        }
        System.out.println(JSON.toJSONString(kws));
    }

    static class TestRun implements Runnable {

        HttpClientDownloader downloader;
        HttpConfig httpConfig;

        public TestRun(HttpClientDownloader downloader, HttpConfig httpConfig) {
            this.downloader = downloader;
            this.httpConfig = httpConfig;
        }

        @Override
        public void run() {

            Map<String, String> params = new LinkedHashMap<>();
            params.put("keyword", "nike");
            params.put("filters", "[]");
            params.put("sort", "");
            params.put("page", "2");
            params.put("page_size", "20");
            params.put("source", "explore_feed");
            params.put("search_id", "5DFA5CF74E17F6815265BD21B64FF84E"); // todo 需破解
//            params.put("search_id",searchId()); // 已破解
            params.put("api_extra", "");
            params.put("page_pos", "20");
            params.put("allow_rewrite", "1");
            params.put("geo", "eyJsYXRpdHVkZSI6NDAuMDAyMDcxLCJsb25naXR1ZGUiOjEwNS41Njc2NTd9%0A");
            params.put("word_request_id", "");
            params.put("platform", "android");
            params.put("deviceId", "403153f8-c986-34fa-9dcb-5194a047cb44");
            params.put("device_fingerprint", "20210205135445ff6e82f3d7517eb9648568126824c6ec01c3d44b07f35b63");
            params.put("device_fingerprint1", "20210205135445ff6e82f3d7517eb9648568126824c6ec01c3d44b07f35b63");
            params.put("versionName", "6.25.0");
            params.put("channel", "Store360");
            params.put("sid", "session.1613627589281279551785");
            params.put("lang", "zh-Hans");
            params.put("t", "1613628282");
            params.put("fid", "16136275431001b693ff0cd0c08e64ca69927611f3d1");
            String sign = genSignature(params, "403153f8-c986-34fa-9dcb-5194a047cb44");
            logger.info(sign);
            params.put("sign", sign); //已破解

            List<NameValuePair> nameValuePairs = new ArrayList<>();

            StringBuffer paramStr = new StringBuffer();
            for (Map.Entry<String, String> entry : params.entrySet()) {
                try {
                    nameValuePairs.add(new BasicNameValuePair(entry.getKey(), URLDecoder.decode(entry.getValue(), "utf-8")));
                } catch (UnsupportedEncodingException e) {
                    e.printStackTrace();
                }
            }
//            params.forEach((key,value) -> {
//                try {
//                    paramStr.append(key).append("=").append(URLEncoder.encode(URLDecoder.decode(value, "utf-8"),"utf-8"));
//                } catch (UnsupportedEncodingException e) {
//                    e.printStackTrace();
//                }
//
//            });

            String url = "https://www.xiaohongshu.com/api/sns/v10/search/notes?" + URLEncodedUtils.format(nameValuePairs, "utf-8");
            HttpRequest httpRequest = new HttpRequest();

            for (int i = 0; i < 1; i++) {

                httpRequest.setUrl(url);
                httpRequest.addHeader("xy-common-params", "platform=android&deviceId=403153f8-c986-34fa-9dcb-5194a047cb44&device_fingerprint=20210205135445ff6e82f3d7517eb9648568126824c6ec01c3d44b07f35b63&device_fingerprint1=20210205135445ff6e82f3d7517eb9648568126824c6ec01c3d44b07f35b63&versionName=6.25.0&channel=Store360&sid=session.1613627589281279551785&t=1613628282&fid=16136275431001b693ff0cd0c08e64ca69927611f3d1&uis=light");
                httpRequest.addHeader("User-Agent", "Dalvik/2.1.0 (Linux; U; Android 7.1.2; MI 9 Build/NMF26X) Resolution/720*1280 Version/6.25.0 Build/6250099 Device/(Xiaomi;MI 9) discover/6.25.0 NetType/WiFi");
                httpRequest.addHeader("shield", "6b4fbbb2020aac9386c923891946bb49920e08da17a77f0775f3989c967a3ddd"); // todo 需破解
                httpRequest.addHeader("xy-platform-info", "platform=android&build=6250099&deviceId=403153f8-c986-34fa-9dcb-5194a047cb44");
                httpRequest.addHeader("Host", "www.xiaohongshu.com");
                httpRequest.addHeader("Connection", "Keep-Alive");
                HttpPage httpPage = downloader.download(httpRequest, httpConfig);
                logger.info("download total count {}",count++);
                logger.info(httpPage.getRawText());

            }
        }
    }

    static class TestUserNoteRun implements Runnable {

        HttpClientDownloader downloader;
        HttpConfig httpConfig;

        public TestUserNoteRun(HttpClientDownloader downloader, HttpConfig httpConfig) {
            this.downloader = downloader;
            this.httpConfig = httpConfig;
        }

        @Override
        public void run() {
            String userNotesFormat = "https://www.xiaohongshu.com/fe_api/burdock/weixin/v2/user/%s/notes?page=%d&page_size=15";
            String userId = "5d0d6ef90000000011009007";
            String userNotes = String.format(userNotesFormat, userId,1);
            String urlPath = userNotes.substring(userNotes.indexOf("/fe_api/"));
            int page = 1;
            try {
                while (true){
                    page++;
                    String x_sign = "X" + md5(urlPath + "WSUDD");
                    HttpRequest httpRequest = new HttpRequest();
                    httpRequest.setUrl(userNotes);
                    httpRequest.setHeaders(headers);
                    httpRequest.addHeader("User-Agent", getOneUA());
                    httpRequest.addHeader("x-sign", x_sign);
                    List<String> data = null;
                    try {
                        HttpPage httpPage = downloader.download(httpRequest, httpConfig);
                        logger.info("download total count {}",count++);
                        Json json = new Json(httpPage.getRawText());
                        data = json.jsonPath("$.data").all();
                        for (String datum : data) {
                            System.out.println(datum);
                        }
                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                    if(data == null || data.size() < 1){
                        break;
                    }
                    userNotes = String.format(userNotesFormat, userId,page);
                    urlPath = userNotes.substring(userNotes.indexOf("/fe_api/"));
                }

            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }


    static class TestSearchRun implements Runnable {

        HttpClientDownloader downloader;
        HttpConfig httpConfig;

        public TestSearchRun(HttpClientDownloader downloader, HttpConfig httpConfig) {
            this.downloader = downloader;
            this.httpConfig = httpConfig;
        }

        @Override
        public void run() {

            String urlFormat = "https://www.xiaohongshu.com/fe_api/burdock/weixin/v2/search/notes?" +
                    "keyword=%s&sortBy=create_time_desc&page=%d&pageSize=20&prependNoteIds=&needGifCover=true";
            RedisReader redisReader = new RedisReader("192.168.1.215", 6379, 2);
            String noteUrlFormat = "https://www.xiaohongshu.com/fe_api/burdock/weixin/v2/note/%s/single_feed";
//            try {
//                int index = 1;
//                while (true) {
//                    CrawlerRequestRecord crawlerRequestRecord = redisReader.recordPop("crawler-xhs-queue_list", "crawler-xhs-queue_map");
//                    if (crawlerRequestRecord == null){
//                        Thread.sleep(3000);
//                        continue;
//                    }
//                    String requestUrl = crawlerRequestRecord.getHttpRequest().getUrl();
//                    List<NameValuePair> parameters = URLEncodedUtils.parse(requestUrl.split("\\?")[1], Charset.forName("utf-8"));
//                    String keyword = "";
//                    for (NameValuePair parameter : parameters) {
//                        if ("keyword".equals(parameter.getName())) {
//                            keyword = parameter.getValue();
//                            break;
//                        }
//                    }
//                    logger.info("current keyword {}",keyword);
//                    String rawText = crawlerRequestRecord.getInternalDownloadPage().getRawText();
//                    Json rawTextJson = new Json(rawText);
//                    List<String> items = null;
//                    try {
//                        items = rawTextJson.jsonPath($_type + ".data.items").all();
//
//                    }catch (Exception e){
//                        logger.error("error search kw {}, content {}",keyword,rawText);
//                        continue;
//                    }
//
//                    if (items != null && items.size() > 0){
//                        for (String item : items) {
//                            Json itemJson = new Json(item);
//                            String modelType = itemJson.jsonPath($_type + ".model_type").get();
//                            if ("note".equals(modelType)) {
//                                String noteId = itemJson.jsonPath($_type + ".note.id").get();
//                                String noteUrl = String.format(noteUrlFormat, noteId);
//                                processNoteArticle(noteId,noteUrl, downloader, httpConfig,itemJson);
//                            }
//                        }
//                    }
//
//                    index++;
//
//                }
//            } catch (Exception e) {
//                e.printStackTrace();
//            }
            String noteUrl = String.format(noteUrlFormat, "60f53faa000000002103c262");
            try {
                processNoteArticle("60f53faa000000002103c262",noteUrl, downloader, httpConfig,null);
            } catch (NoSuchAlgorithmException e) {
                e.printStackTrace();
            }
        }




        private void processNoteArticle(String noteId,String noteUrl, HttpClientDownloader downloader, HttpConfig httpConfig,Json itemJson) throws NoSuchAlgorithmException {
            String urlPath = noteUrl.substring(noteUrl.indexOf("/fe_api/"));

            String x_sign = "X" + md5(urlPath + "WSUDD");
            logger.info("note " + noteUrl + " x_sign:" + x_sign);
//
//            try {
//                Random rand = new Random();
//                long sleepTime = (long)((rand.nextFloat()+0.5)*2000L);
//                logger.info("xhs article sleep time {}",sleepTime);
//                Thread.sleep(sleepTime);
//            } catch (InterruptedException e) {
//                e.printStackTrace();
//            }

//            String nikeName = itemJson.jsonPath($_type + ".note.user.nickname").get();
//            String type = itemJson.jsonPath($_type + ".note.type").get();
//            String id = itemJson.jsonPath($_type + ".note.id").get();
//            String userid = itemJson.jsonPath($_type + ".note.user.userid").get();

//            tracker(downloader, httpConfig,"NoteDetail",id,nikeName,type,userid);

            HttpRequest httpRequest = new HttpRequest();
            httpRequest.setUrl(noteUrl);
            httpRequest.setHeaders(headers);
            httpRequest.addHeader("User-Agent", getOneUA());
            httpRequest.addHeader("x-sign", x_sign);
            HttpPage httpPage = downloader.download(httpRequest, httpConfig);
            logger.info("download total count {}",count++);
            logger.info("article detail : {}", httpPage.getRawText());

//            List<String> all = itemJson.jsonPath($_type+ ".note.images_list[*].url").all();
//            all.add(itemJson.jsonPath($_type + ".note.user.images").get());
//            images(downloader,all);

//            banner(downloader,httpConfig,id);
//            related(downloader,httpConfig,id);
//            mpActivityBanner(downloader,httpConfig);

//            try{
//                Json notePage = new Json(httpPage.getRawText());
////                String noteId = notePage.jsonPath(".data.id").get();
//                int comments = Integer.valueOf(notePage.jsonPath(".data.comments").get());
//                if (comments > 0) {
//                    processNoteArticleComment(noteId, downloader, httpConfig);
//                }
//            }catch (Exception e){
//                e.printStackTrace();
//            }

//            processNoteArticleComment(noteId, downloader, httpConfig);
        }

        private void processNoteArticleComment (String noteId, HttpClientDownloader downloader, HttpConfig httpConfig) throws NoSuchAlgorithmException {
            String endId = null;
            String noteCommentFirst = "https://www.xiaohongshu.com/fe_api/burdock/weixin/v2/notes/%s/comments?pageSize=10";
            String noteCommentOther = "https://www.xiaohongshu.com/fe_api/burdock/weixin/v2/notes/%s/comments?pageSize=10&endId=%s";
            String noteComment;
            while (true) {
                if (StringUtils.isNotBlank(endId)) {
                    noteComment = String.format(noteCommentOther, noteId, endId);
                } else {
                    noteComment = String.format(noteCommentFirst, noteId);
                }

                String urlPath = noteComment.substring(noteComment.indexOf("/fe_api/"));

                String x_sign = "X" + md5(urlPath + "WSUDD");
                logger.info("note id " + noteId + " comment x_sign:" + x_sign);

                HttpRequest httpRequest = new HttpRequest();
                httpRequest.setUrl(noteComment);
                httpRequest.setHeaders(headers);
                httpRequest.addHeader("User-Agent", getOneUA());
                httpRequest.addHeader("x-sign", x_sign);

                try {
                    Random rand = new Random();
                    long sleepTime = (long)((rand.nextFloat()+1)*2000L);
                    logger.info("xhs article comment sleep time {}",sleepTime);
                    Thread.sleep(sleepTime);


//                    collect(downloader, httpConfig,"NoteCommentDetail",noteId);
                    HttpPage httpPage = downloader.download(httpRequest, httpConfig);

                    logger.info("download total count {}",count++);
                    Json noteCommentPage = new Json(httpPage.getRawText());
                    String code = noteCommentPage.jsonPath("$.code").get();
                    if (!code.equals("0")){
                        logger.info("xhs article comment error content {}",httpPage.getRawText());
                        break;
                    }
                    List<String> noteComments = noteCommentPage.jsonPath(".data.comments.*").all();
                    if (noteComments == null || noteComments.size() < 1) {
                        break;
                    }
                    for (String comment : noteComments) {
                        logger.info("article comment detail : {}", comment);
                        Json commentJson = new Json(comment);
                        endId = commentJson.jsonPath("$.id").get();
                    }

                    servicewechat(downloader, httpConfig);
                    images(downloader);
                } catch (Exception e) {
                    break;
                }
            }
        }

        private void tracker(HttpClientDownloader downloader,HttpConfig httpConfig,String crawlerType,String id
                ,String nikeName,String type,String userid){

            HttpRequest httpRequest = new HttpRequest();
            httpRequest.setUrl("https://www.xiaohongshu.com/fe_api/burdock/weixin/v2/device/record");
            httpRequest.setHeaders(headers);
            httpRequest.addHeader("content-type","application/json");
            httpRequest.addHeader("x-sign","X51cda0aa5b5e64b7aee826084ec164fa");
            httpRequest.addHeader("User-Agent", getOneUA());
            httpRequest.setMethod("POST");
            String body = "{\"noteId\":\""+id+"\",\"pasteboard\":\"\"}";

            httpRequest.setRequestBody(HttpRequestBody.json(body,"utf-8"));
            HttpPage download = downloader.download(httpRequest, httpConfig);
            System.out.println(download.getRawText());

        }

        private void banner(HttpClientDownloader downloader,HttpConfig httpConfig,String id) throws NoSuchAlgorithmException {
            String url = "https://www.xiaohongshu.com/fe_api/burdock/weixin/v2/banner?noteId="+id+"&platform=weixin";
            String urlPath = url.substring(url.indexOf("/fe_api/"));

            String x_sign = "X" + md5(urlPath + "WSUDD");

            HttpRequest httpRequest = new HttpRequest();
            httpRequest.setUrl(url);
            httpRequest.setHeaders(headers);
            httpRequest.addHeader("content-type","application/json");
            httpRequest.addHeader("x-sign",x_sign);
            httpRequest.addHeader("User-Agent", getOneUA());

            HttpPage download = downloader.download(httpRequest, httpConfig);
            System.out.println(download.getRawText());

        }

        private void related(HttpClientDownloader downloader,HttpConfig httpConfig,String id) throws NoSuchAlgorithmException {

            String url = "https://www.xiaohongshu.com/fe_api/burdock/weixin/v2/note/"+id+"/related?openId=oPPUS5eJr92sMqanyhemFu6mw314&pageSize=8&page=1&noteType=1&needGifCover=true";
            String urlPath = url.substring(url.indexOf("/fe_api/"));

            String x_sign = "X" + md5(urlPath + "WSUDD");

            HttpRequest httpRequest = new HttpRequest();
            httpRequest.setUrl("https://www.xiaohongshu.com/fe_api/burdock/weixin/v2/note/"+id+"/related?openId=oPPUS5eJr92sMqanyhemFu6mw314&pageSize=8&page=1&noteType=1&needGifCover=true");
            httpRequest.setHeaders(headers);
            httpRequest.addHeader("content-type","application/json");
            httpRequest.addHeader("x-sign",x_sign);
            httpRequest.addHeader("User-Agent", getOneUA());

            HttpPage download = downloader.download(httpRequest, httpConfig);
            System.out.println(download.getRawText());

        }

        private void mpActivityBanner(HttpClientDownloader downloader,HttpConfig httpConfig){

            HttpRequest httpRequest = new HttpRequest();
            httpRequest.setUrl("https://pages.xiaohongshu.com/data/sns/mp_activity_banner?sid=session.1614320808394350242858");
//        httpRequest.setHeaders(headers);
//        httpRequest.addHeader("content-type","application/json");
//        httpRequest.addHeader("User-Agent", getOneUA());
//        httpRequest.addHeader("Host", "pages.xiaohongshu.com");

            HttpPage download = downloader.download(httpRequest, httpConfig);
            System.out.println(download.getRawText());

        }

        private void servicewechat(HttpClientDownloader downloader,HttpConfig httpConfig) throws NoSuchAlgorithmException {
            String url = "https://servicewechat.com/ci.xiaohongshu.com/2b8b5668-64ea-471a-86e2-969e9883f43f";
            HttpRequest httpRequest = new HttpRequest();
            httpRequest.setUrl(url);
            downloader.download(httpRequest, httpConfig);
        }

        private void images(HttpClientDownloader downloader) throws NoSuchAlgorithmException {
            HttpRequest httpRequest = new HttpRequest();
            HttpConfig httpConfig = HttpConfig.me("image");
            httpConfig.setResponseTextGenerateHtml(false);
            for (String url : imageList) {
                httpRequest.setUrl(url);

                downloader.download(httpRequest, httpConfig);
            }
        }

        private void images(HttpClientDownloader downloader,List<String> imageList) throws NoSuchAlgorithmException {
            HttpRequest httpRequest = new HttpRequest();
            HttpConfig httpConfig = HttpConfig.me("image");
            httpConfig.setResponseTextGenerateHtml(false);
            for (String url : imageList) {
                if(StringUtils.isNotBlank(url)){
                    httpRequest.setUrl(url);

                    downloader.download(httpRequest, httpConfig);
                }

            }
        }

    }



    private static void collect(HttpClientDownloader downloader,HttpConfig httpConfig,String crawlerType,String id){
            String eid = UUID.randomUUID().toString();

            HttpRequest httpRequest = new HttpRequest();
            httpRequest.setUrl("https://t.xiaohongshu.com/api/collect");
            httpRequest.setHeaders(headers);
            httpRequest.addHeader("User-Agent", getOneUA());
            httpRequest.setMethod("POST");
            String bodyFormat = "{\"schema\":\"payload\",\"data\":[{\"tna\":\"mpT\",\"tv\":\"mp-0.1.0\",\"aid\":\"xhs_marco_mewtwo#7.33.1\",\"vp\":\"360x550\",\"p\":\"mp\",\"e\":\"se\",\"exp\":\"jcao_wechat_6250_2|jcao_wechat_5639_2|yuchenwang_wechat_4355_2|jcao_wechat_8066_2|jcao_wechat_6340_2|dji2_wechat_2632_4|yuchenwang_wechat_4217_3|jcao_wechat_5711_2\",\"eid\":\"%s\",\"se_ca\":\"%s\",\"se_ac\":\"pv\",\"uid\":\"cb649af686d3210a726cb4fc33a241bb\",\"nuid\":\"603894a8000000000100132f\",\"url\":\"%s\",\"co\":\"{\\\"schema\\\":\\\"xhs_mini_program/1-0-0\\\",\\\"data\\\":[{\\\"schema\\\":\\\"mp-user/1-0-0\\\",\\\"data\\\":{\\\"openid\\\":\\\"oPPUS5eJr92sMqanyhemFu6mw314\\\",\\\"isNewWxmpUser\\\":false,\\\"isAppUser\\\":false,\\\"wxNickname\\\":\\\"\\\",\\\"sid\\\":\\\"session.1614320808394350242858\\\",\\\"authorization\\\":\\\"wxmp.2018fb5f-d900-4e0f-9e9a-df358614fed5\\\"}},{\\\"schema\\\":\\\"page-info/1-0-0\\\",\\\"data\\\":{\\\"from\\\":\\\"engagebar\\\",\\\"last_scene\\\":1089,\\\"can_launch_app\\\":false,\\\"scene\\\":1089}},{\\\"schema\\\":\\\"system-info/1-0-0\\\",\\\"data\\\":{\\\"model\\\":\\\"DIG-AL00\\\",\\\"platform\\\":\\\"android\\\",\\\"sdk\\\":\\\"2.6.6\\\",\\\"system\\\":\\\"Android 6.0\\\",\\\"version\\\":\\\"7.0.3\\\"}}]}\",\"se_pr\":\"%s\",\"se_la\":1089}]}";


            String body = null;
            if (crawlerType.equals("NoteDetail")){
//                String articleUrlFormat = "pages/main/note/index?nickname=%s&type=%s&id=%s&userid=%s&scene=1089";
//                String articleUrl = String.format(articleUrlFormat,nikeName,type,id,userid);
//                body = String.format(bodyFormat,eid,crawlerType,articleUrl,id);
            } else {
                String articleCommentUrlFormat = "pages/secondary/comment/index?from=engagebar&id=%s&scene=1089";
                String articleCommentUrl = String.format(articleCommentUrlFormat,id);
                body = String.format(bodyFormat,eid,crawlerType,articleCommentUrl,id);
            }
            httpRequest.setRequestBody(HttpRequestBody.json(body,"utf-8"));
            downloader.download(httpRequest,httpConfig);
    }


    private static String getOneUA(){
        if (index >= useragents.size()){
            index = 0;
        }
//        String ua = useragents.get(index)+" MicroMessenger/7.0.3.1400(0x27000334) Process/appbrand2 NetType/WIFI Language/zh_CN";
        String ua = "Mozilla/5.0 (Linux; Android 6.0; DIG-AL00 Build/HUAWEIDIG-AL00; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/55.0.2883.91 Mobile Safari/537.36 MicroMessenger/7.0.3.1400(0x27000334) Process/appbrand0 NetType/WIFI Language/zh_CN";
        index++;
        return ua;
    }

        public static String genSignature (Map < String, String > params, String deviceId){
            if (params == null) {
                return null;
            }
            String sign = "";
            StringBuffer sb = new StringBuffer();
            try {
                // 1. 字典升序排序
                SortedMap<String, String> sortedMap = new TreeMap<>(params);

                Set<String> keySet = sortedMap.keySet();
                for (String key : keySet) {
                    //sign不参与算法
                    if (key.equals("sig")) {
                        continue;
                    }
                    String value = sortedMap.get(key);
                    sb.append(key + "=" + URLDecoder.decode(value, "UTF-8"));
                }
                String uriString = sb.toString();
                logger.info("uriString --> " + uriString);

                //2 URL ENCODE
                String encodeParams = URLEncoder.encode(uriString, "utf-8");
                logger.info("encodeParams --> " + encodeParams);

                //3 TO BYTE ARRAY
                byte[] paramsBytes = encodeParams.getBytes();

                //4 deviceId to byte array
                byte[] deviceIdBytes = deviceId.getBytes();

                //5 对3和4的结果每个字节进行相关的异或运算
                StringBuffer sbs = new StringBuffer();
                int i = 0;
                for (byte paramsByte : paramsBytes) {
                    sbs.append(Byte.valueOf((byte) (paramsByte ^ deviceIdBytes[i])));
                    i = (i + 1) % deviceIdBytes.length;
                }
                logger.info("sbs --> " + sbs.toString());


                logger.info("xor_str --> " + sbs);

                // 6 MD5运算得到请求签名
                sign = md5(sbs);
                logger.info("first sign --> " + sign);

                sign = md5(sign + deviceId);
                logger.info("second sign --> " + sign);
            } catch (Exception e) {
                logger.error(e.getMessage(), e);
            }
            return sign;
        }

        public static String genSignatureForWx (String urlPath) throws NoSuchAlgorithmException {
            final String secrtKey = "WSUDD";

            return md5(urlPath + secrtKey);
        }


        public static String searchId () {
            StringBuilder sb = new StringBuilder();
            sb.append(System.currentTimeMillis());
            String format = String.format("%064d", Arrays.copyOf(new Object[]{Long.valueOf((long) (Math.random() * 9.223372036854776E18d))}, 1));
            sb.append(format);
            String b2 = b(sb.toString());
            return b2;
        }

        public static String b (String str){
            try {
                return a(str.getBytes("utf-8"));
            } catch (UnsupportedEncodingException e2) {
                e2.printStackTrace();
                return "";
            }
        }

        public static String a ( byte[] bArr){
            try {
                MessageDigest instance = MessageDigest.getInstance("MD5");
                instance.update(bArr);
                return aha(instance.digest());
            } catch (NoSuchAlgorithmException e2) {
                e2.printStackTrace();
                return "";
            }
        }

        public static String aha ( byte[] bArr){
            if (bArr == null || bArr.length == 0) {
                return "";
            }
            int length = bArr.length;
            char[] cArr = new char[(length * 2)];
            for (int i2 = 0; i2 < length; i2++) {
                byte b2 = bArr[i2];
                int i3 = i2 * 2;
                char[] cArr2 = f52734c;
                cArr[i3] = cArr2[(b2 >>> 4) & 15];
                cArr[i3 + 1] = cArr2[b2 & 15];
            }
            return new String(cArr);
        }


}
